package org.liuh.designs.spider;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.liuh.designs.spider.dtos.AddressNode;
import org.liuh.designs.spider.utils.HttpClientUtil;
import org.liuh.designs.spider.utils.RedisUtils;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;

/**
 * @ClassName: SpiderHelper
 * @Description: 区划代码和城乡区划代码
 * @Author: liuh
 * @Date: 2020/9/29 16:58
 * @Version: 1.0
 */
public class SpiderAddressHelper {

    private final static String path = "D:\\";

    private static RedisUtils redisUtils = new RedisUtils();

    /**
     * @Description: test获取数据..
     * @Author: liuh
     * @Date: 2020/9/29
     */
    public static void main(String[] args) throws Exception {
        HttpClientUtil.resetProxyIP(100, 100);
        long startTime = System.currentTimeMillis();
        ThreadPoolManager poolManager = ThreadPoolManager.getInstance();
        // 不设置最大线程池.. 使用环境cpu..
        ExecutorService threadPool = poolManager.getCachedThreadPool();
        // 切换请求为 httpClient..
        AreaConstant.HttpUtilEnum httpClient = AreaConstant.HttpUtilEnum.HTTP_CLIENT;
        // 获取省份.
        AddressNode rootNode = AreaConstant.getRootNode(httpClient);
        long provinceEndTime = System.currentTimeMillis();
        System.out.println(String.format("获取所有的省所用时间： %s 秒", (provinceEndTime - startTime) / 1000));
        if (rootNode == null) {
            System.out.println("IP 又被封了~ 省： 0个");
            return;
        }
        CountDownLatch provinceDownLatch = new CountDownLatch(rootNode.getChildNodeList().size());
        List<AddressNode> provinceList = rootNode.getChildNodeList();
        // 保存地址文件：
        for (AddressNode province : provinceList) {
            // 抓取各省份的市
            AreaSpiderProvinceTask task = new AreaSpiderProvinceTask(province, provinceDownLatch, httpClient);
            threadPool.submit(task);
        }
        provinceDownLatch.await();
        System.out.println("------------------------所有省信息开始-------------------------");
        JSONArray provinceJSONArray = new JSONArray();
        JSONObject objProvince;
        for (AddressNode addressNode : rootNode.getChildNodeList()) {
            objProvince = new JSONObject();
            objProvince.put("code", addressNode.getAddressCode());
            objProvince.put("name", addressNode.getAddressName());
            objProvince.put("url", addressNode.getNodeUrl());
            if (addressNode.getChildNodeList() != null) {
                objProvince.put("childSize", addressNode.getChildNodeList().size());
            }
            provinceJSONArray.add(objProvince);
        }
        // 添加到Redis..中..
        redisUtils.setnx(AreaConstant.RDS_PROVINCE_KEY, JSON.toJSONString(provinceJSONArray));
        System.out.println("------------------------所有省信息结束-------------------------");
        long provinceTime = System.currentTimeMillis();
        System.out.printf("获取所有的市所用时间：%s 秒 \n", (provinceTime - startTime) / 1000);
        // 获取所有的县/区
        List<AddressNode> cityNodeList = new ArrayList<>();
        for (AddressNode province : provinceList) {
            cityNodeList.addAll(province.getChildNodeList());
        }
        System.out.println(String.format("市数量： %s", cityNodeList.size()));
        long cityTimeBegin = System.currentTimeMillis();
        CountDownLatch cityDown = new CountDownLatch(cityNodeList.size());
        for (AddressNode cityNode : cityNodeList) {
            AreaSpiderCityTask task = new AreaSpiderCityTask(cityNode, cityDown, httpClient);
            threadPool.submit(task);
        }
        cityDown.await();
        System.out.println("------------------------所有市信息开始-------------------------");
        JSONArray cityJSONArray = new JSONArray();
        JSONObject objCity;
        for (AddressNode addressNode : cityNodeList) {
            objCity = new JSONObject();
            objCity.put("code", addressNode.getAddressCode());
            objCity.put("name", addressNode.getAddressName());
            objCity.put("url", addressNode.getNodeUrl());
            if (addressNode.getParentNode() != null) {
                objCity.put("provinceCode", addressNode.getParentNode().getAddressCode());
                objCity.put("provinceName", addressNode.getParentNode().getAddressName());
            }
            if (addressNode.getChildNodeList() != null) {
                objCity.put("childSize", addressNode.getChildNodeList().size());
            }
            cityJSONArray.add(objCity);
        }
        redisUtils.setnx(AreaConstant.RDS_CITY_KEY, JSON.toJSONString(cityJSONArray));
        System.out.println("------------------------所有市信息结束-------------------------");
        long cityTimeEnd = System.currentTimeMillis();
        System.out.printf("获取所有的区县所用时间：%s 秒 \n", (cityTimeEnd - cityTimeBegin) / 1000);
        // 所有区县..
        List<AddressNode> countyNodeList = new ArrayList<>();
        for (AddressNode cityNode : cityNodeList) {
            if (cityNode.getChildNodeList() != null) {
                countyNodeList.addAll(cityNode.getChildNodeList());
            }
        }
        System.out.println("------------------------所有区/县信息开始-------------------------");
        JSONArray arrayCounty = new JSONArray();
        JSONObject objCounty;
        for (AddressNode addressNode : countyNodeList) {
            objCounty = new JSONObject();
            objCounty.put("code", addressNode.getAddressCode());
            objCounty.put("name", addressNode.getAddressName());
            objCounty.put("url", addressNode.getNodeUrl());
            if (addressNode.getParentNode() != null) {
                objCounty.put("cityCode", addressNode.getParentNode().getAddressCode());
                objCounty.put("cityName", addressNode.getParentNode().getAddressName());
                if (addressNode.getParentNode().getParentNode() != null) {
                    objCounty.put("provinceCode", addressNode.getParentNode().getParentNode().getAddressCode());
                    objCounty.put("provinceName", addressNode.getParentNode().getParentNode().getAddressName());
                }
            }
//            if (addressNode.getChildNodeList() != null) {
//                obj.put("childSize", addressNode.getChildNodeList().size());
//            }
            arrayCounty.add(objCounty);
        }
        redisUtils.setnx(AreaConstant.RDS_CITY_KEY, JSON.toJSONString(arrayCounty));
        System.out.println(String.format("json区县数量： %s", arrayCounty.size()));
        System.out.println(arrayCounty);
        System.out.println("------------------------所有区/县信息结束-------------------------");
        System.out.println(String.format("区县数量： %s", countyNodeList.size()));

        //TODO IO操作..
//        String countyJson = JSONObject.toJSONString(countyNodeList);
//        FileUtils.createJsonFile(countyJson, path, "countyJson");
//        System.out.println(String.format("区县数量： %s", countyNodeList.size()));
//        ExecutorService countyTaskPool = poolManager.getSingleScheduledThreadPool(6);
//        // 切换ip
//        // 获取街道..
//
//        long countyTimeBegin = System.currentTimeMillis();
//        CountDownLatch countyDown = new CountDownLatch(countyNodeList.size());//countyNodeList.size());
//        for (AddressNode countyNode : countyNodeList) {
//            AreaSpiderCountyTask task = new AreaSpiderCountyTask(countyNode, countyDown, httpClient);
//            countyTaskPool.submit(task);
//        }
//        countyDown.await();
//        long countyTimeEnd = System.currentTimeMillis();
//        System.out.printf("获取所有的街道所用时间：%s 秒 \n", (countyTimeEnd - countyTimeBegin) / 1000);
//        List<AddressNode> townNodeList = new ArrayList<>();
//        for (AddressNode countyNode : countyNodeList) {
//            if (countyNode.getChildNodeList() != null) {
//                townNodeList.addAll(countyNode.getChildNodeList());
//            }
//        }
//        System.out.println(String.format("街道数量： %s", townNodeList.size()));

        // 持久化操作..
        // TODO 1. 保存省份信息..
        // TODO 2. 保存市信息..
        // TODO 3. 保存区/县信息..
        // TODO 4. 保存街道信息..
        // 结束进程..
//        countyTaskPool.shutdown();
        threadPool.shutdown();
        long totalEndTime = System.currentTimeMillis();
        System.out.println(String.format("总耗时： %s", (totalEndTime - startTime) / 1000));
    }

}
